package cn.edu.hznu.web.crawl;

import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Optional;

import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public class CrawlLiteratureBooks {
	
	final static String ROOTURL = "http://douban.com/tag/";
	final static String FILEDIR = "/Users/LVZY/Documents/Record/Crawl/douban.com/tag/tmp/";
	
	public void crawl() {
//		String indexResponse = DownloadContent.getStrContentFromUrl(ROOTURL, "utf-8");
//		Document indexPageDoc = Jsoup.parse(indexResponse);
//		Element LiteraturePartElm = indexPageDoc.getElementsByAttributeValue("name", "文学")
//												.first();
//		LiteraturePartElm.siblingElements()
//						 .first()
//						 .getElementsByTag("a")
//						 .forEach(tagElm -> {
//							 File currentTagFolder = this.createFolder(ROOTURL + tagElm.text() + "/");
//							 try (BufferedWriter bfWriter =
//									 this.createBufferedWriter(currentTagFolder + "/book-list.txt")) {
//								 this.processBookListPageContent(currentTagFolder, bfWriter, 
//										 ROOTURL + tagElm.text() + "/book?start=", 0);
//							 } catch (IOException e) {
//								 e.printStackTrace();
//							 }
//						 });
		
		
		File currentTagFolder = this.createFolder(FILEDIR + "张爱玲/");
		try (BufferedWriter bfWriter = this.createBufferedWriter(currentTagFolder + "/book-list.txt")) {
			this.processBookListPageContent(currentTagFolder, bfWriter, ROOTURL + "张爱玲/book?start=", 0);
		} catch (IOException e) {
			e.printStackTrace();
		}
		
	}
	
	private void processBookListPageContent(
					 File folder, BufferedWriter bfWriter, String commonUrl, int page) {
		System.out.println("BookList---->Crawl " + String.valueOf(page) + " page of " + "张爱玲");
		
		// Rest 3 seconds after the end of pre page
		try {
			Thread.sleep(2000);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		try {
			Document bookListDoc;
			bookListDoc = Jsoup.parse(
						DownloadContent.getStrContentFromUrl(commonUrl + String.valueOf(page), "utf-8"));
			Optional.ofNullable(bookListDoc.getElementsByTag("dt"))
					.ifPresent(bookLinkElm -> {
						bookLinkElm.forEach(bookElm -> {
							
							// Rest 3 seconds after the end of pre book
							try {
								Thread.sleep(3000);
							} catch (Exception e) {
								e.printStackTrace();
							}
							
							this.processBookPageStrContent(folder, bfWriter, 
									bookElm.getElementsByTag("a").first().attr("href"));
						});
						if (bookLinkElm.size() == 15) {
							this.processBookListPageContent(folder, bfWriter, commonUrl, page + 15);
						}
			});
		} catch (ParseException | IOException e1) {
			e1.printStackTrace();
		}
	}
	
	private void processBookPageStrContent(File folder, BufferedWriter bfWriter, String url) {
		
		System.out.println("BookDetails---->Cawl at url: " + url);
		
		try {
			Document bookDetailsDoc;
			bookDetailsDoc = Jsoup.parse(DownloadContent.getStrContentFromUrl(url, "utf-8"));
			Optional.ofNullable(bookDetailsDoc.getElementById("info"))
					.ifPresent(infoElm -> {
						this.writeUsingBfWriterWithExcption(bfWriter, infoElm.text(), 1);
					});
					Optional.ofNullable(bookDetailsDoc.getElementsByClass("related_info").first())
							.ifPresent(relatedInfoElm -> {
								Optional.ofNullable(relatedInfoElm.getElementsByTag("h2").first())
										.ifPresent(h2Elm -> {
											this.writeUsingBfWriterWithExcption(bfWriter, h2Elm.text(), 1);
										});
								Optional.ofNullable(relatedInfoElm.getElementsByClass("intro").first())
										.ifPresent(introElm -> {
											this.writeUsingBfWriterWithExcption(bfWriter, introElm.text(), 2);
										});
					});
			
			Optional.ofNullable(bookDetailsDoc.getElementById("mainpic"))
					.ifPresent(imgElm -> {
						Optional.ofNullable(imgElm.getElementsByTag("a").first())
								.ifPresent(aElm -> {
									String[] splitedImgUrl = aElm.attr("href").split("/");
									this.writeImg(
											folder, 
											aElm.attr("title") + splitedImgUrl[splitedImgUrl.length - 1], 
											aElm.attr("href"));
								});
					});
		} catch (ParseException | IOException e) {
			e.printStackTrace();
		}
	}
	
	private void writeUsingBfWriterWithExcption(BufferedWriter bfWriter, String text, int newLineCount) {
		try {
			bfWriter.write(text);
			while (newLineCount-- > 0) {
				 bfWriter.newLine();
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void writeImg(File folder, String imgName, String imgUrl) {
		
		System.out.println("BookImg---->Crawl at url: " + imgUrl);
		
		try {
			byte[] responseByte = DownloadContent.getByteContentFromUrl(imgUrl);
			try (BufferedOutputStream bfOutStream = 
					new BufferedOutputStream(
							new FileOutputStream(
									new File(folder, imgName)))) {
				bfOutStream.write(responseByte);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private File createFolder(String folderName) {
		File folder = new File(folderName);
		if (!folder.exists()) {
			folder.mkdirs();
		}
		return folder;
	}
	
	private BufferedWriter createBufferedWriter(String filename) 
			throws IOException {
		return new BufferedWriter(
					new FileWriter(filename));
	}
	
}
